# -*- coding:utf-8 -*-
# @Time : 2020/9/22 9:52
# @Author : liruiyang
# @File : spider_main
# @Software: PyCharm
# @Description: 这是一个python程序
# here put the import lib
import json
import random
import requests
from bs4 import BeautifulSoup
import bs4
from spider_app import config
from spider_app.models.model import InformationCode, InformationInfo


def code_spider(url):
    response = requests.get(url, headers=headers)
    soup = BeautifulSoup(response.text, "html5lib")
    codes = soup.find_all("a", target="_top")
    code_url_list = []
    for code in codes:
        result_dict = InformationCode.add_code_info(**{"codeName": code.text.strip(), "parentCodeID": "20201114185739"})
        if result_dict.get("code") != "200":
            return result_dict
        code_url_list.append((result_dict.get("data"),
                              base_url + (code.attrs.get("href")[6:] if "/java/" in code.attrs.get("href") else code.attrs.get("href"))
                              ))
    return code_url_list


def text_spider(code_url):
    url = code_url[1]
    response = requests.get(url, headers=headers)
    soup = BeautifulSoup(response.text, "html5lib")
    codes = soup.find("div", id="content")
    text_list = []
    for content in codes.contents:
        if isinstance(content, bs4.element.NavigableString):
            continue
        text_list.append((content.name, content.text.strip()))
    text_json = json.dumps(text_list, ensure_ascii=False)
    InformationInfo.add_detail_info(**{"title": code_url[0].get("codeName"),
                                       "text": text_json,
                                       "parentCodeID": code_url[0].get("codeID")})
    return text_json


if __name__ == '__main__':
    headers = {
        'Accept': 'application/json, text/javascript, */*; q=0.01',
        'Accept-Encoding': 'gzip, deflate, compress',
        'Accept-Language': 'en-us;q=0.5,en;q=0.3',
        'Cache-Control': 'max-age=0',
        'Connection': 'keep-alive',
        'User-Agent': config.HEAD_LIST[random.randint(0, 20)]
    }
    base_url = "https://www.runoob.com/java/"
    first_url = base_url + "java-tutorial.html"
    code_url_list = code_spider(first_url)
    for code_url in code_url_list:
        text_spider(code_url)
    print("爬取完成")